package zxk.spider.webmagic.processor;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.lang.Console;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import zxk.spider.domain.MovieEntity;

import java.util.ArrayList;
import java.util.List;

/**
 * @fileName: PatentPageProcessor
 * @author: 舟小亢
 * @date: 2022-11-6 21:56:38
 * @description: 美剧天堂页面爬取
 */
@Component
public class MoviePageProcessor implements PageProcessor {

    /**
     * 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private Site site = Site.me()
            .setRetryTimes(5)
            .setSleepTime(1000)
            .setTimeOut(20 * 1000);

    /**
     * process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
     *
     * @return
     */
    @Override
    public void process(Page page) {
        //获取页面
        Html html = page.getHtml();

        //获取列表
        Selectable init = html.css("div[class='list3_cn_box ']");

        List<String> urlList;
        if (!init.all().isEmpty()) {
            urlList = new ArrayList<>();
            //列表页面,获取列表添加进队列
            List<Selectable> nodes = init.css("div.cn_box2 ").nodes();
            for (Selectable node : nodes) {
                String cn_box2 = node.css("div.cn_box_box3 div.bor_img3_right a", "href").toString();
                if (cn_box2 != null) {
                    urlList.add("https://www.meijutt.tv" + cn_box2);
                }
            }
            //详细页面地址
            page.addTargetRequests(urlList);

            //抓取下一页
            StringBuilder nextPage = new StringBuilder("https://www.meijutt.tv");
            Selectable page1 = init.css(".page").nodes().get(0);
            //span的text按/分割
            String text = page1.css("span", "text").toString();
            String[] split = text.trim().split("/");
            if (split[0].equals("1")) {
                //如果前==1，则在首页
                nextPage = nextPage.append(page1.css("a").nodes().get(6).css("a", "href"));
                page.addTargetRequest(nextPage.toString());
            } else if (!split[0].equals(split[1])) {
                //如果前后不相等，则不在尾页，继续加载
                nextPage = nextPage.append(page1.css("a").nodes().get(8).css("a", "href"));
                page.addTargetRequest(nextPage.toString());
            }
        } else {
            //详细页面表单数据
            List<Selectable> nodes = html.css("div.info-box div.o_r_contact_all div.o_r_contact ul>li").nodes();

            //包装实体
            MovieEntity m = new MovieEntity();
            m.setMovieName(nodes.get(1).css("li", "text").toString());
            m.setAliasName(nodes.get(2).css("li", "text").toString());
            m.setPublishDate(nodes.get(6).css("li", "text").toString());

            //小分类
            StringBuilder sb = new StringBuilder();
            List<Selectable> nodes1 = nodes.get(8).css("li").css("label a").nodes();
            for (Selectable s : nodes1) {
                String str = s.css("a", "text").toString();
                sb.append(",");
                sb.append(str);
            }
            //删除多余逗号
            if (sb.length() >= 1) {
                sb.deleteCharAt(0);
            }
            m.setClassificationDetail(sb.toString());

            m.setRegion(nodes.get(9).css("li label", "text").toString());
            m.setUpdateCycle(nodes.get(9).css("li span font b span", "text").toString());
            m.setTvStation(nodes.get(10).css("li label", "text").toString());
            m.setFilmLength(nodes.get(10).css("li span", "text").toString());
            m.setLatestUpdateTime(DateUtil.parseDateTime(nodes.get(11).css("li label", "text").toString()));
            m.setClassification(nodes.get(11).css("li span", "text").toString());
            m.setPictureUrl(html.css("div.info-box div.o_list div.o_big_img_bg_b img", "src").toString());
            m.setFilmUrl(html.getDocument().location());

            page.putField("tvInfo", m);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        String url = "https://www.meijutt.tv/1_______.html";

        Spider.create(new MoviePageProcessor())
                //开始抓取的位置
                .addUrl(url)
                //开启队列加去重（最大支持10万条数据）
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                //开启线程
                .thread(5)
                //启动爬虫
                .run();
    }
}